{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "AFIchtHPxrM5"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import sys\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import urllib\n",
    "import time\n",
    "FLAGS = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1342,
     "status": "ok",
     "timestamp": 1529336128810,
     "user": {
      "displayName": "Lip Gallagher",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "113091702821929511633"
     },
     "user_tz": -480
    },
    "id": "1aYBNpD-zglv",
    "outputId": "4e9198d2-3451-4919-d215-16d2a31fbd7d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "data_dir = './Desktop/Data/week_06'\n",
    "mnist = input_data.read_data_sets('./',source_url='http://yann.lecun.com/exdb/mnist/', one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FjT1WThmzka2"
   },
   "outputs": [],
   "source": [
    "def weight_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial, collections=[tf.GraphKeys.GLOBAL_VARIABLES,'Weights'])\n",
    "def bias_variable(shape):\n",
    "    initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "    return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FwguWk_zznsZ"
   },
   "outputs": [],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])#\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "bM-jS-lCzp-k"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "with tf.name_scope('conv1'):\n",
    "    w_conv1 = weight_variable([5,5,1,32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    l_conv1 = tf.nn.conv2d(x_image, w_conv1, strides=[1,1,1,1],\n",
    "                          padding = 'SAME') + b_conv1\n",
    "    # 激活函数\n",
    "    h_conv1 = tf.nn.relu(l_conv1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "jBJMAAk8zr-A"
   },
   "outputs": [],
   "source": [
    "# 28*28 - 14*14(32)\n",
    "with tf.name_scope('pool1'):\n",
    "    h_pool1 = tf.nn.max_pool(h_conv1, ksize=[1,2,2,1],\n",
    "                            strides =[1,2,2,1], padding='VALID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "FSGuftHAzuA6"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('conv2'):\n",
    "    w_conv2 = weight_variable([5,5,32,64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    l_conv2 = tf.nn.conv2d(h_pool1, w_conv2, strides=[1,1,1,1],\n",
    "                          padding='SAME') + b_conv2\n",
    "    h_conv2 = tf.nn.relu(l_conv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "ebphhYCszvtS"
   },
   "outputs": [],
   "source": [
    "keep_prob = tf.placeholder(tf.float32) #\n",
    "h_conv2_drop = tf.nn.dropout(h_conv2, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "rEJLfOPbzxq-"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('pool2'):\n",
    "    h_pool2 = tf.nn.max_pool(h_conv2_drop, ksize=[1,2,2,1],\n",
    "                            strides=[1,2,2,1], padding='SAME')\n",
    "# 14x14x64 -->  7x7x64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Plh29ot5zzwu"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('fc1'):\n",
    "    w_fc1 = weight_variable([7*7*64, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1,7*7*64])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w_fc1) + b_fc1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "wB9OGzioz5dt"
   },
   "outputs": [],
   "source": [
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "v7YFjNdsz7WB"
   },
   "outputs": [],
   "source": [
    "with tf.name_scope('fc2'):\n",
    "    w_fc2 = weight_variable([1024,10])\n",
    "    b_fc2 = bias_variable([10])\n",
    "    y = tf.matmul(h_fc1_drop, w_fc2) + b_fc2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "d0YaC8aAz82C"
   },
   "outputs": [],
   "source": [
    "decay_rate = 0.96\n",
    "\n",
    "decay_steps = 1000\n",
    "\n",
    "global_ = tf.Variable(tf.constant(0))\n",
    "\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))\n",
    "\n",
    "#l2_loss = tf.add_n([tf.nn.l2_loss(w) for w in tf.get_collection('Weights')])\n",
    "\n",
    "#total_loss = cross_entropy + 7e-5*l2_loss\n",
    "\n",
    "\n",
    "\n",
    "#updateparameter = tf.group(update_parameter, update_parameter2)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_, 1))\n",
    "\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "lr = tf.train.exponential_decay(0.001, global_, 1000,0.98 ) # (learning_rate, global, decay_step, decay_rate)\n",
    "train_step = tf.train.GradientDescentOptimizer(lr).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 3635
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6136085,
     "status": "ok",
     "timestamp": 1529347850894,
     "user": {
      "displayName": "Lip Gallagher",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "113091702821929511633"
     },
     "user_tz": -480
    },
    "id": "KTSpS4qXz-rm",
    "outputId": "97fa5494-a68c-43f0-874d-bf82493b9f46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100 : entropy loss: 0.711211, learning_rate: 0.00999006\n",
      "step 200 : entropy loss: 0.566893, learning_rate: 0.00998002\n",
      "step 300 : entropy loss: 0.287287, learning_rate: 0.00996999\n",
      "step 400 : entropy loss: 0.242912, learning_rate: 0.00995998\n",
      "step 500 : entropy loss: 0.311784, learning_rate: 0.00994997\n",
      "step 600 : entropy loss: 0.27775, learning_rate: 0.00993998\n",
      "step 700 : entropy loss: 0.14802, learning_rate: 0.00992999\n",
      "step 800 : entropy loss: 0.206473, learning_rate: 0.00992002\n",
      "step 900 : entropy loss: 0.16988, learning_rate: 0.00991005\n",
      "step 1000 : entropy loss: 0.362292, learning_rate: 0.0099001\n",
      "step 1100 : entropy loss: 0.238861, learning_rate: 0.00989015\n",
      "step 1200 : entropy loss: 0.210926, learning_rate: 0.00988022\n",
      "step 1300 : entropy loss: 0.180248, learning_rate: 0.00987029\n",
      "step 1400 : entropy loss: 0.218859, learning_rate: 0.00986038\n",
      "step 1500 : entropy loss: 0.0909929, learning_rate: 0.00985047\n",
      "step 1600 : entropy loss: 0.13047, learning_rate: 0.00984058\n",
      "step 1700 : entropy loss: 0.105174, learning_rate: 0.00983069\n",
      "step 1800 : entropy loss: 0.119868, learning_rate: 0.00982082\n",
      "step 1900 : entropy loss: 0.124574, learning_rate: 0.00981095\n",
      "step 2000 : entropy loss: 0.157641, learning_rate: 0.0098011\n",
      "step 2100 : entropy loss: 0.124188, learning_rate: 0.00979125\n",
      "step 2200 : entropy loss: 0.0967752, learning_rate: 0.00978142\n",
      "step 2300 : entropy loss: 0.261478, learning_rate: 0.00977159\n",
      "step 2400 : entropy loss: 0.099799, learning_rate: 0.00976178\n",
      "step 2500 : entropy loss: 0.0784737, learning_rate: 0.00975197\n",
      "step 2600 : entropy loss: 0.0857208, learning_rate: 0.00974217\n",
      "step 2700 : entropy loss: 0.0651788, learning_rate: 0.00973239\n",
      "step 2800 : entropy loss: 0.096673, learning_rate: 0.00972261\n",
      "step 2900 : entropy loss: 0.194825, learning_rate: 0.00971284\n",
      "step 3000 : entropy loss: 0.175819, learning_rate: 0.00970309\n",
      "step 3100 : entropy loss: 0.0693874, learning_rate: 0.00969334\n",
      "step 3200 : entropy loss: 0.0691316, learning_rate: 0.0096836\n",
      "step 3300 : entropy loss: 0.0219652, learning_rate: 0.00967388\n",
      "step 3400 : entropy loss: 0.0249898, learning_rate: 0.00966416\n",
      "step 3500 : entropy loss: 0.0979975, learning_rate: 0.00965445\n",
      "step 3600 : entropy loss: 0.203099, learning_rate: 0.00964475\n",
      "step 3700 : entropy loss: 0.182391, learning_rate: 0.00963506\n",
      "step 3800 : entropy loss: 0.0748922, learning_rate: 0.00962539\n",
      "step 3900 : entropy loss: 0.0576498, learning_rate: 0.00961572\n",
      "step 4000 : entropy loss: 0.0268752, learning_rate: 0.00960606\n",
      "step 4100 : entropy loss: 0.134086, learning_rate: 0.00959641\n",
      "step 4200 : entropy loss: 0.0124328, learning_rate: 0.00958677\n",
      "step 4300 : entropy loss: 0.0229224, learning_rate: 0.00957714\n",
      "step 4400 : entropy loss: 0.0281497, learning_rate: 0.00956752\n",
      "step 4500 : entropy loss: 0.0475205, learning_rate: 0.00955791\n",
      "step 4600 : entropy loss: 0.0623345, learning_rate: 0.0095483\n",
      "step 4700 : entropy loss: 0.0542225, learning_rate: 0.00953871\n",
      "step 4800 : entropy loss: 0.0663108, learning_rate: 0.00952913\n",
      "step 4900 : entropy loss: 0.0339584, learning_rate: 0.00951956\n",
      "step 5000 : entropy loss: 0.15125, learning_rate: 0.00951\n",
      "step 5100 : entropy loss: 0.0839006, learning_rate: 0.00950044\n",
      "step 5200 : entropy loss: 0.053622, learning_rate: 0.0094909\n",
      "step 5300 : entropy loss: 0.0428274, learning_rate: 0.00948137\n",
      "step 5400 : entropy loss: 0.0596613, learning_rate: 0.00947184\n",
      "step 5500 : entropy loss: 0.0760974, learning_rate: 0.00946233\n",
      "step 5600 : entropy loss: 0.0868933, learning_rate: 0.00945282\n",
      "step 5700 : entropy loss: 0.0711248, learning_rate: 0.00944333\n",
      "step 5800 : entropy loss: 0.167002, learning_rate: 0.00943384\n",
      "step 5900 : entropy loss: 0.122961, learning_rate: 0.00942436\n",
      "step 6000 : entropy loss: 0.201027, learning_rate: 0.0094149\n",
      "step 6100 : entropy loss: 0.120747, learning_rate: 0.00940544\n",
      "step 6200 : entropy loss: 0.0337616, learning_rate: 0.00939599\n",
      "step 6300 : entropy loss: 0.0417592, learning_rate: 0.00938655\n",
      "step 6400 : entropy loss: 0.0688119, learning_rate: 0.00937712\n",
      "step 6500 : entropy loss: 0.0497503, learning_rate: 0.0093677\n",
      "step 6600 : entropy loss: 0.0270817, learning_rate: 0.00935829\n",
      "step 6700 : entropy loss: 0.0907859, learning_rate: 0.00934889\n",
      "step 6800 : entropy loss: 0.0386344, learning_rate: 0.0093395\n",
      "step 6900 : entropy loss: 0.235209, learning_rate: 0.00933012\n",
      "step 7000 : entropy loss: 0.0390575, learning_rate: 0.00932075\n",
      "step 7100 : entropy loss: 0.111264, learning_rate: 0.00931138\n",
      "step 7200 : entropy loss: 0.115887, learning_rate: 0.00930203\n",
      "step 7300 : entropy loss: 0.0866795, learning_rate: 0.00929269\n",
      "step 7400 : entropy loss: 0.127039, learning_rate: 0.00928335\n",
      "step 7500 : entropy loss: 0.0123816, learning_rate: 0.00927403\n",
      "step 7600 : entropy loss: 0.0468183, learning_rate: 0.00926471\n",
      "step 7700 : entropy loss: 0.099083, learning_rate: 0.0092554\n",
      "step 7800 : entropy loss: 0.0451036, learning_rate: 0.00924611\n",
      "step 7900 : entropy loss: 0.022338, learning_rate: 0.00923682\n",
      "step 8000 : entropy loss: 0.0846701, learning_rate: 0.00922754\n",
      "step 8100 : entropy loss: 0.117585, learning_rate: 0.00921827\n",
      "step 8200 : entropy loss: 0.0214543, learning_rate: 0.00920901\n",
      "step 8300 : entropy loss: 0.0857209, learning_rate: 0.00919976\n",
      "step 8400 : entropy loss: 0.0286499, learning_rate: 0.00919052\n",
      "step 8500 : entropy loss: 0.0341076, learning_rate: 0.00918129\n",
      "step 8600 : entropy loss: 0.0643659, learning_rate: 0.00917206\n",
      "step 8700 : entropy loss: 0.056309, learning_rate: 0.00916285\n",
      "step 8800 : entropy loss: 0.0285443, learning_rate: 0.00915365\n",
      "step 8900 : entropy loss: 0.101903, learning_rate: 0.00914445\n",
      "step 9000 : entropy loss: 0.051411, learning_rate: 0.00913526\n",
      "step 9100 : entropy loss: 0.0231731, learning_rate: 0.00912609\n",
      "step 9200 : entropy loss: 0.0810744, learning_rate: 0.00911692\n",
      "step 9300 : entropy loss: 0.0106487, learning_rate: 0.00910776\n",
      "step 9400 : entropy loss: 0.0618339, learning_rate: 0.00909861\n",
      "step 9500 : entropy loss: 0.0826866, learning_rate: 0.00908947\n",
      "step 9600 : entropy loss: 0.11821, learning_rate: 0.00908034\n",
      "step 9700 : entropy loss: 0.0894983, learning_rate: 0.00907122\n",
      "step 9800 : entropy loss: 0.130578, learning_rate: 0.00906211\n",
      "step 9900 : entropy loss: 0.0340695, learning_rate: 0.00905301\n",
      "step 10000 : entropy loss: 0.160724, learning_rate: 0.00904391\n",
      "step 10100 : entropy loss: 0.0233325, learning_rate: 0.00903483\n",
      "step 10200 : entropy loss: 0.0258571, learning_rate: 0.00902575\n",
      "step 10300 : entropy loss: 0.0229009, learning_rate: 0.00901669\n",
      "step 10400 : entropy loss: 0.0272144, learning_rate: 0.00900763\n",
      "step 10500 : entropy loss: 0.0417569, learning_rate: 0.00899858\n",
      "step 10600 : entropy loss: 0.0438529, learning_rate: 0.00898954\n",
      "step 10700 : entropy loss: 0.0365383, learning_rate: 0.00898051\n",
      "step 10800 : entropy loss: 0.0462859, learning_rate: 0.00897149\n",
      "step 10900 : entropy loss: 0.0625087, learning_rate: 0.00896248\n",
      "step 11000 : entropy loss: 0.0764415, learning_rate: 0.00895347\n",
      "step 11100 : entropy loss: 0.0384418, learning_rate: 0.00894448\n",
      "step 11200 : entropy loss: 0.0456308, learning_rate: 0.00893549\n",
      "step 11300 : entropy loss: 0.0100662, learning_rate: 0.00892652\n",
      "step 11400 : entropy loss: 0.0364152, learning_rate: 0.00891755\n",
      "step 11500 : entropy loss: 0.0564276, learning_rate: 0.00890859\n",
      "step 11600 : entropy loss: 0.0606502, learning_rate: 0.00889964\n",
      "step 11700 : entropy loss: 0.0580041, learning_rate: 0.0088907\n",
      "step 11800 : entropy loss: 0.108603, learning_rate: 0.00888177\n",
      "step 11900 : entropy loss: 0.0708731, learning_rate: 0.00887285\n",
      "step 12000 : entropy loss: 0.0142685, learning_rate: 0.00886394\n",
      "step 12100 : entropy loss: 0.0936923, learning_rate: 0.00885503\n",
      "step 12200 : entropy loss: 0.00594899, learning_rate: 0.00884614\n",
      "step 12300 : entropy loss: 0.0341671, learning_rate: 0.00883725\n",
      "step 12400 : entropy loss: 0.0393656, learning_rate: 0.00882838\n",
      "step 12500 : entropy loss: 0.0240045, learning_rate: 0.00881951\n",
      "step 12600 : entropy loss: 0.0881573, learning_rate: 0.00881065\n",
      "step 12700 : entropy loss: 0.090879, learning_rate: 0.0088018\n",
      "step 12800 : entropy loss: 0.0669298, learning_rate: 0.00879296\n",
      "step 12900 : entropy loss: 0.0333466, learning_rate: 0.00878412\n",
      "step 13000 : entropy loss: 0.0399032, learning_rate: 0.0087753\n",
      "step 13100 : entropy loss: 0.0458247, learning_rate: 0.00876648\n",
      "step 13200 : entropy loss: 0.108356, learning_rate: 0.00875768\n",
      "step 13300 : entropy loss: 0.0702812, learning_rate: 0.00874888\n",
      "step 13400 : entropy loss: 0.0393894, learning_rate: 0.00874009\n",
      "step 13500 : entropy loss: 0.0651133, learning_rate: 0.00873131\n",
      "step 13600 : entropy loss: 0.0441, learning_rate: 0.00872254\n",
      "step 13700 : entropy loss: 0.0254949, learning_rate: 0.00871378\n",
      "step 13800 : entropy loss: 0.0702697, learning_rate: 0.00870503\n",
      "step 13900 : entropy loss: 0.0124373, learning_rate: 0.00869628\n",
      "step 14000 : entropy loss: 0.0264237, learning_rate: 0.00868755\n",
      "step 14100 : entropy loss: 0.0399677, learning_rate: 0.00867882\n",
      "step 14200 : entropy loss: 0.0606396, learning_rate: 0.0086701\n",
      "step 14300 : entropy loss: 0.0303664, learning_rate: 0.00866139\n",
      "step 14400 : entropy loss: 0.0615558, learning_rate: 0.00865269\n",
      "step 14500 : entropy loss: 0.01528, learning_rate: 0.008644\n",
      "step 14600 : entropy loss: 0.0224639, learning_rate: 0.00863532\n",
      "step 14700 : entropy loss: 0.048152, learning_rate: 0.00862664\n",
      "step 14800 : entropy loss: 0.0191173, learning_rate: 0.00861798\n",
      "step 14900 : entropy loss: 0.0326353, learning_rate: 0.00860932\n",
      "step 15000 : entropy loss: 0.0179713, learning_rate: 0.00860067\n",
      "step 15100 : entropy loss: 0.0684673, learning_rate: 0.00859203\n",
      "step 15200 : entropy loss: 0.017627, learning_rate: 0.0085834\n",
      "step 15300 : entropy loss: 0.204083, learning_rate: 0.00857478\n",
      "step 15400 : entropy loss: 0.0661586, learning_rate: 0.00856616\n",
      "step 15500 : entropy loss: 0.0616659, learning_rate: 0.00855756\n",
      "step 15600 : entropy loss: 0.0515343, learning_rate: 0.00854896\n",
      "step 15700 : entropy loss: 0.0385238, learning_rate: 0.00854038\n",
      "step 15800 : entropy loss: 0.0502672, learning_rate: 0.0085318\n",
      "step 15900 : entropy loss: 0.00833597, learning_rate: 0.00852323\n",
      "step 16000 : entropy loss: 0.0325463, learning_rate: 0.00851466\n",
      "step 16100 : entropy loss: 0.0360977, learning_rate: 0.00850611\n",
      "step 16200 : entropy loss: 0.0467623, learning_rate: 0.00849757\n",
      "step 16300 : entropy loss: 0.0754708, learning_rate: 0.00848903\n",
      "step 16400 : entropy loss: 0.0509554, learning_rate: 0.0084805\n",
      "step 16500 : entropy loss: 0.0791114, learning_rate: 0.00847198\n",
      "step 16600 : entropy loss: 0.077497, learning_rate: 0.00846347\n",
      "step 16700 : entropy loss: 0.00173684, learning_rate: 0.00845497\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 16800 : entropy loss: 0.0193883, learning_rate: 0.00844648\n",
      "step 16900 : entropy loss: 0.034612, learning_rate: 0.00843799\n",
      "step 17000 : entropy loss: 0.0298235, learning_rate: 0.00842952\n",
      "step 17100 : entropy loss: 0.0101591, learning_rate: 0.00842105\n",
      "step 17200 : entropy loss: 0.0531376, learning_rate: 0.00841259\n",
      "step 17300 : entropy loss: 0.0104308, learning_rate: 0.00840414\n",
      "step 17400 : entropy loss: 0.0135131, learning_rate: 0.0083957\n",
      "step 17500 : entropy loss: 0.0702436, learning_rate: 0.00838726\n",
      "step 17600 : entropy loss: 0.0889653, learning_rate: 0.00837884\n",
      "step 17700 : entropy loss: 0.0484689, learning_rate: 0.00837042\n",
      "step 17800 : entropy loss: 0.0377512, learning_rate: 0.00836201\n",
      "step 17900 : entropy loss: 0.0535332, learning_rate: 0.00835361\n",
      "step 18000 : entropy loss: 0.00796673, learning_rate: 0.00834522\n",
      "step 18100 : entropy loss: 0.0383258, learning_rate: 0.00833684\n",
      "step 18200 : entropy loss: 0.0366692, learning_rate: 0.00832846\n",
      "step 18300 : entropy loss: 0.0122454, learning_rate: 0.0083201\n",
      "step 18400 : entropy loss: 0.0188322, learning_rate: 0.00831174\n",
      "step 18500 : entropy loss: 0.0367638, learning_rate: 0.00830339\n",
      "step 18600 : entropy loss: 0.103662, learning_rate: 0.00829505\n",
      "step 18700 : entropy loss: 0.011985, learning_rate: 0.00828672\n",
      "step 18800 : entropy loss: 0.00747192, learning_rate: 0.00827839\n",
      "step 18900 : entropy loss: 0.0188869, learning_rate: 0.00827008\n",
      "step 19000 : entropy loss: 0.0701544, learning_rate: 0.00826177\n",
      "step 19100 : entropy loss: 0.013362, learning_rate: 0.00825347\n",
      "step 19200 : entropy loss: 0.0205884, learning_rate: 0.00824518\n",
      "step 19300 : entropy loss: 0.0255003, learning_rate: 0.0082369\n",
      "step 19400 : entropy loss: 0.0407748, learning_rate: 0.00822862\n",
      "step 19500 : entropy loss: 0.0372626, learning_rate: 0.00822036\n",
      "step 19600 : entropy loss: 0.0264601, learning_rate: 0.0082121\n",
      "step 19700 : entropy loss: 0.0412924, learning_rate: 0.00820385\n",
      "step 19800 : entropy loss: 0.0218079, learning_rate: 0.00819561\n",
      "step 19900 : entropy loss: 0.0612479, learning_rate: 0.00818738\n",
      "step 20000 : entropy loss: 0.00642524, learning_rate: 0.00817915\n",
      "test accuracy 0.9862\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    sess.run(init_op)\n",
    "    for step in range(20000):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "        ts, loss, l, = sess.run([train_step, cross_entropy, lr],\n",
    "                                 feed_dict={x:batch_xs, y_:batch_ys, keep_prob: 0.75, global_: step})\n",
    "        test_acc = sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                            y_: mnist.test.labels, keep_prob:0.75})\n",
    "        if (step+1) % 100==0:\n",
    "            print('step %d : entropy loss: %g, learning_rate: %g' %(step+1, loss, l))\n",
    "    print('test accuracy %g' % (test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "TseGjjjW0Uv_"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "Untitled0.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
